import downloader
from db import category_col
from utils import get_site, md5

if __name__ == '__main__':
    url = 'https://www.3yt.org/'
    resp = downloader.download(url)
    lis = resp.xpath('//div[@class="nav"]/ul/li')[1:]
    base_url = get_site(resp.url)
    for li in lis:
        name = li.xpath('a/text()').extract_first()
        href = li.xpath('a/@href').extract_first()
        if href:
            category = {
                'site': url,
                'url':  base_url + href,
                'name': name,
                'md5': md5(href)
            }
            category_col.update_one({'md5': category['md5']}, {'$set': category}, upsert=True)
